﻿

#include <fstream>
#include <ctype.h>
#include <cmath>
#include <map>
#include <list>
#include <stdlib.h>
//#include <py_api/py_runner.h>
#include "tdx_file.h"
#include <cppbase/utility.h>
#include <cppbase/bitset_type.h>
#include "file_store.h"

namespace fs = std::filesystem;
MyFileStore *MyFileStore::inst = nullptr;
/*
void MyFileStore::set_bit(char* bit, int cnt)
{
	char* targetbyte = bit + cnt / 8;
	*targetbyte = (*targetbyte) | (1 << (cnt % 8));
}
*/
dynamic_bitset<> MyFileStore::get_bitmap(std::map<uint32_t, std::vector<MinuteBar>> *bars)
{
	dynamic_bitset<> ret;

	uint32_t begintime = bars->begin()->first;
	ret.resize(DateTime::diff_days(begintime, bars->rbegin()->first) + 1, false);

	for (auto i = bars->begin(); i != bars->end(); i++)
	{
		ret[DateTime::diff_days(begintime, i->first)] = true;
	}
	return ret;
}

dynamic_bitset<> MyFileStore::get_bitmap(std::map<uint32_t, Bar> *bars)
{
	dynamic_bitset<> ret;

	uint32_t begintime = bars->begin()->first;
	ret.resize(DateTime::diff_days(begintime, bars->rbegin()->first) + 1, false);

	for (auto i = bars->begin(); i != bars->end(); i++)
	{
		ret[DateTime::diff_days(begintime, i->first)] = true;
	}
	return ret;
}
dynamic_bitset<> MyFileStore::get_bitmap(Bar *buf, int cnt)
{
	dynamic_bitset<> ret;

	uint32_t begintime = buf->time;
	uint32_t endtime = (buf + cnt - 1)->time;
	ret.resize(DateTime::diff_days(begintime, endtime) + 1);
	for (int i = 0; i < cnt; i++)
	{
		ret[DateTime::diff_days(begintime, (((Bar *)buf) + i)->time)] = true;
	}
	return ret;
}
dynamic_bitset<> MyFileStore::get_bitmap(std::vector<Bar> *bars)
{
	dynamic_bitset<> ret;

	uint32_t begintime = bars->begin()->time;
	ret.resize(DateTime::diff_days(begintime, bars->rbegin()->time) + 1);

	for (auto i = bars->begin(); i != bars->end(); i++)
	{
		ret[DateTime::diff_days(begintime, i->time)] = true;
	}
	return ret;
}
dynamic_bitset<> MyFileStore::get_bitmap(std::set<Bar> bars)
{
	dynamic_bitset<> ret;

	uint32_t begintime = bars.begin()->time;
	ret.resize(DateTime::diff_days(begintime, bars.rbegin()->time) + 1, false);

	for (auto i = bars.begin(); i != bars.end(); i++)
	{
		ret[DateTime::diff_days(begintime, i->time)] = true;
	}
	return ret;
}
void MyFileStore::save_trading_date()
{
	auto fpath = fs::path(dir) / trading_date_fn;
	std::ofstream of;
	of.open(fpath, std::ios::out | std::ios::trunc);
	std::map<uint32_t, bool> datelist;
	for (auto &i : tradingdate)
	{
		datelist[i.first] = true;
	}
	for (auto &i : datelist)
	{
		std::string s = std::to_string(i.first);
		of.write(s.data(), 8);
		of.write("\n", 1);
	}
	of.close();
}

void MyFileStore::write_bars(std::fstream &of, std::map<uint32_t, std::vector<MinuteBar>> *bars, std::map<uint32_t, std::vector<MinuteBar>>::iterator start_it)
{
	int barperday = std::max(bars->begin()->second.size(), bars->rbegin()->second.size());
	for (; start_it != bars->end(); start_it++)
	{

		of.write((char *)(&start_it->first), 4);
		of.write((char *)(start_it->second.data()), start_it->second.size() * sizeof(MinuteBar));
		if (start_it->second.size() < barperday)
		{
			int diff = barperday - start_it->second.size();
			MinuteBar dmp;
			dmp.min = -1;
			while (diff != 0)
			{
				of.write((char *)(&dmp), sizeof(MinuteBar));
				diff--;
			}
		}
	}
}

//bitmap,startdate(4),lastdate(4),rowsize(2),footsize(2)
/*
std::vector<char> MyFileStore::get_bitmap_and_foot(std::map<uint32_t, std::vector<MinuteBar>>* bars)
{
	std::vector<char> ret;

	auto v=get_bitmap(bars);
uint16_t row_size = bars->begin()->second.size() *sizeof(MinuteBar)+sizeof(uint32_t);
uint16_t footsize = v.total_bytes() + sizeof(uint32_t)*2+sizeof(uint16_t) * 2;
ret.resize(footsize);
//bit map
memcpy(ret.data(), v.data(), v.total_bytes());
//first date
memcpy(ret.data()+ v.total_bytes(), &(bars->begin()->first),4);
//last date
memcpy(ret.data() + v.total_bytes()+4, &(bars->rbegin()->first), 4);
//row size(bytes per row)
memcpy(ret.data() + v.total_bytes()+8, &row_size, 2);
memcpy(ret.data() + v.total_bytes() + 10, &footsize, 2);

return ret;

}
*/
void MyFileStore::print_bar_info(Exchange e, std::string symbol, BarSize bs)
{
	auto p = get_filepath(e, symbol, bs);
	if (!fs::exists(p))
	{
		//	LOG(INFO)<<std::printf("file(%s) not exist.",symbol.data());
		return;
	}
	//LOG(INFO) << p.string()<<", size:"<<fs::file_size(p)<<" bytes.";

	std::map<uint32_t, Bar> bars;
	std::map<uint32_t, std::vector<MinuteBar>> minbars;
	dynamic_bitset<> bitmap;
	BarFileFoot foot;

	if (bs == BarSize::Day1)
	{

		read_daybar(p, &bars, &bitmap, &foot);
	}
	else
	{
		read_minutebar(p, &minbars, &bitmap, &foot);
	}
	print_bitmap(&bitmap);
	print_foot(&foot);

	if (bs == BarSize::Day1)
	{

		print_rows(&bars);
	}
	else
	{
		print_rows(&minbars);
	}
}
void MyFileStore::print_foot(BarFileFoot *foot)
{

	std::cout << "\nBar File Foot Info:\n"
			  << "\tFirst Date:" << DateTime(foot->firstdate).to_day()
			  << "\tLast Date: " << DateTime(foot->lastdate).to_day()
			  << "\tRow Size(bytes): " << foot->rowsize
			  << "\tFoot Size(12 bytes of tail):" << foot->footsize << std::endl;
}
void MyFileStore::print_bitmap(dynamic_bitset<> *bitmap)
{

	std::cout << "\nBitmap Info:\n"
			  << "\tblocks(64 bytes each) count:" << bitmap->num_blocks()
			  << "\tFirst block content:" << *(uint64_t *)bitmap->data()
			  << "\tLast block content:" << *(uint64_t *)(bitmap->data() + bitmap->num_blocks() - 1) << std::endl;
}
void MyFileStore::print_rows(std::map<uint32_t, std::vector<MinuteBar>> *b)
{
	std::cout << "Date\n\tMin,Open,High,Low,Close,Volume,Amout\n";

	if (b->size() <= 2)
	{
		for (auto &i : *b)
		{

			std::cout << DateTime(i.first).to_day() << std::endl;
			for (auto &v : i.second)
			{
				std::cout << v.min << "," << v.open << "," << v.high << "," << v.low << "," << v.close << "," << v.volume << "," << v.amount << std::endl;
			}
		}
	}
	else
	{

		std::cout << DateTime(b->begin()->first).to_day() << std::endl;
		for (auto &v : b->begin()->second)
		{
			std::cout << v.min << "," << v.open << "," << v.high << "," << v.low << "," << v.close << "," << v.volume << "," << v.amount << std::endl;
		}
		std::cout << "... ...\n";
		std::cout << DateTime(b->rbegin()->first).to_day() << std::endl;
		for (auto &v : b->rbegin()->second)
		{
			std::cout << v.min << "," << v.open << "," << v.high << "," << v.low << "," << v.close << "," << v.volume << "," << v.amount << std::endl;
		}
	}
}
void MyFileStore::print_rows(std::map<uint32_t, Bar> *b)
{
	std::cout << "Date,Open,High,Low,Close,Volume,Amout\n";
	if (b->size() <= 6)
	{

		for (auto &i : *b)
		{
			std::cout << DateTime(i.second.time).to_day() << "," << i.second.open << "," << i.second.high << "," << i.second.low << "," << i.second.close << "," << i.second.volume << "," << i.second.amount << std::endl;
		}
	}
	else
	{
		auto beg = b->begin();
		int rows = 6;
		Bar *dbar;
		for (int i = 0; i < rows; i++)
		{
			dbar = &beg->second;
			std::cout << DateTime(dbar->time).to_day() << "," << dbar->open << "," << dbar->high << "," << dbar->low << "," << dbar->close << "," << dbar->volume << "," << dbar->amount << std::endl;

			if (i == rows / 2 - 1)
			{
				std::cout << "... ...\n";
				std::advance(beg, b->size() - rows + 1);
			}
			else
			{
				std::advance(beg, 1);
			}
		}
	}
}
void MyFileStore::change_format()
{
	auto path = get_filepath(Exchange::SZ, "000001", BarSize::Min1).parent_path();

	for (auto &p : fs::directory_iterator(path))
	{
		std::fstream f;

		BarFileFoot foot;
		std::map<uint32_t, std::vector<MinuteBar>> bars, nbars;
		read_minutebar(p.path(), &bars, nullptr, &foot);
		for (auto &i : bars)
		{
			nbars[DateTime::day2time(i.first)] = std::move(i.second);
		}
		auto bitmap = get_bitmap(&nbars);

		f.open(p.path(), std::ios::binary | std::ios::out | std::ios::in);
		f.seekg(0, std::ios::end);
		int rowcnt = nbars.size();
		auto beg = nbars.begin();
		for (int i = 0; i < rowcnt; i++)
		{
			f.seekp(i * foot.rowsize, std::ios::beg);
			f.write((char *)(&beg->first), 4);
			beg++;
		}
		update_foot(&nbars, &bitmap, &foot);
		f.seekp(-foot.footsize, std::ios::end);
		f.write((char *)bitmap.data(), bitmap.total_bytes());
		f.write((char *)(&foot), sizeof(BarFileFoot));
		f.close();
	}
}
void MyFileStore::change_daybar_format()
{
	auto path = get_filepath(Exchange::SH, "000001", BarSize::Day1).parent_path();

	for (auto &p : fs::directory_iterator(path))
	{
		std::fstream f;
		auto bytes = read_bytes(p.path());
		auto barmap = unpack_daybar(&bytes);
		auto bitset = get_bitmap(&barmap);
		BarFileFoot foot;
		foot.firstdate = barmap.begin()->first;
		foot.lastdate = barmap.rbegin()->first;
		foot.rowsize = sizeof(Bar);
		foot.footsize = sizeof(BarFileFoot) + bitset.total_bytes();
		f.open(p.path(), std::ios::out | std::ios::in | std::ios::binary);
		f.seekp(-foot.footsize, std::ios::end);
		f.write((char *)(bitset.data()), bitset.total_bytes());
		f.write((char *)(&foot), sizeof(BarFileFoot));
		f.close();
	}
}
#pragma pack(push, 2)
struct oldBar
{
	uint32_t date;
	uint32_t open;
	uint32_t high;
	uint32_t low;
	uint32_t close;
	uint64_t volume;
	uint64_t amount;
};
#pragma pack(pop)

#pragma pack(push, 2)
struct oldMinBar
{
	BarDateTime date;
	float open;
	float high;
	float low;
	float close;
	uint32_t volume;
	uint32_t amount;
};
#pragma pack(pop)

void MyFileStore::move_from()
{

	std::string from = "C:\\Users\\Y\\Documents\\code\\qt_project\\StockAnalyzer\\data\\k\\SH\\1m";
	fs::path frompath = fs::path(from);
	auto targetpath = fs::path(dir) / "SH" / "1m";
	fs::create_directories(targetpath);
	for (auto &p : fs::directory_iterator(frompath))
	{
		if (fs::exists(targetpath / p.path().filename().string().substr(0, 6)))
			continue;
		std::ifstream f;
		f.open(p.path(), std::ios::binary | std::ios::in);
		if (!f.is_open())
		{
			std::cout << "open failed:" << p.path() << std::endl;
			return;
		}
		f.seekg(0, std::ios::end);
		uint32_t size = f.tellg();
		int row = (size - 32) / 28;
		f.seekg(32, std::ios::beg);
		oldMinBar mbar;
		std::map<uint32_t, std::vector<MinuteBar>> bars;
		for (int i = 0; i < row; i++)
		{
			f.read((char *)&mbar, 28);
			uint32_t second = DateTime::day2time(mbar.date.date());
			if (i % 240 == 0)
			{
				bars[second].resize(240);
			}
			bars[second][i % 240].open = round(mbar.open * 100);
			bars[second][i % 240].high = round(mbar.high * 100);
			bars[second][i % 240].low = round(mbar.low * 100);
			bars[second][i % 240].close = round(mbar.close * 100);
			bars[second][i % 240].amount = mbar.amount;
			bars[second][i % 240].volume = mbar.volume;
			bars[second][i % 240].min = mbar.date.min();
		}
		f.close();
		auto bitmap = get_bitmap(&bars);
		BarFileFoot foot;
		foot.firstdate = bars.begin()->first;
		foot.lastdate = bars.rbegin()->first;
		foot.rowsize = 4 + sizeof(MinuteBar) * 240;
		foot.footsize = sizeof(BarFileFoot) + bitmap.total_bytes();

		std::ofstream of;
		of.open(targetpath / p.path().filename().string().substr(0, 6), std::ios::binary | std::ios::out | std::ios::trunc);
		for (auto &i : bars)
		{
			uint32_t t = i.first;
			of.write((char *)&t, 4);
			of.write((char *)i.second.data(), 240 * sizeof(MinuteBar));
		}
		of.write((char *)bitmap.data(), bitmap.total_bytes());
		of.write((char *)&foot, sizeof(BarFileFoot));
		of.close();
	}
}
void MyFileStore::move_from_day()
{
	std::string from = "C:\\Users\\Y\\Documents\\code\\qt_project\\StockAnalyzer\\data\\k\\INDX\\d1";
	fs::path frompath = fs::path(from);
	auto targetpath = fs::path(dir) / "IND" / "1d";
	fs::create_directories(targetpath);
	for (auto &p : fs::directory_iterator(frompath))
	{
		if (fs::exists(targetpath / p.path().filename().string().substr(0, 6)))
			continue;
		std::ifstream f;
		f.open(p.path(), std::ios::binary | std::ios::in);
		if (!f.is_open())
		{
			std::cout << "open failed:" << p.path() << std::endl;
			return;
		}
		f.seekg(0, std::ios::end);
		uint32_t size = f.tellg();
		int row = (size) / 36;
		if (row < 1)
			continue;
		f.seekg(0, std::ios::beg);
		oldBar mbar;
		std::map<uint32_t, Bar> bars;
		for (int i = 0; i < row; i++)
		{
			f.read((char *)&mbar, 36);
			uint32_t second = DateTime::day2time(mbar.date);
			bars[second].open = mbar.open;
			bars[second].high = mbar.high;
			bars[second].low = mbar.low;
			bars[second].close = mbar.close;
			bars[second].amount = mbar.amount * 1.0;
			bars[second].volume = mbar.volume;
			bars[second].time = second;
		}
		f.close();
		auto bitmap = get_bitmap(&bars);
		BarFileFoot foot;
		foot.firstdate = bars.begin()->first;
		foot.lastdate = bars.rbegin()->first;
		foot.rowsize = sizeof(Bar);
		foot.footsize = sizeof(BarFileFoot) + bitmap.total_bytes();

		std::ofstream of;
		of.open(targetpath / p.path().filename().string().substr(0, 6), std::ios::binary | std::ios::out | std::ios::trunc);
		for (auto &i : bars)
		{
			of.write((char *)&i.second, sizeof(Bar));
		}
		of.write((char *)bitmap.data(), bitmap.total_bytes());
		of.write((char *)&foot, sizeof(BarFileFoot));
		of.close();
	}
}
std::vector<MinuteBar> MyFileStore::select_fullday_minbar(Exchange e, std::string symbol, uint32_t date)
{
	std::vector<MinuteBar> v;
	if (tradingdate.size() < 1)
		std::terminate();

	BarFileFoot foot;
	auto f = get_filepath(e, symbol, BarSize::Min1);
	if (!fs::exists(f))
	{
		std::cout << "file not exist:" + f.string() << std::endl;
		return v;
	}
	dynamic_bitset<> bitmap;
	read_bitmap_foot(f, &bitmap, &foot);
	auto rowindex = bitmap.count(DateTime::diff_days(foot.firstdate, date)) - bitmap.find_first();
	if (rowindex < 0)
		return v;
	v.resize((foot.rowsize - 4) / sizeof(MinuteBar));
	read_partial_bytes(f, rowindex * foot.rowsize + 4, v.size() * sizeof(MinuteBar), (char *)v.data());
	return v;
}
void MyFileStore::read_minutebar(fs::path f, std::map<uint32_t, std::vector<MinuteBar>> *bars, dynamic_bitset<> *bitmap, BarFileFoot *foot)
{
	if (!foot)
		return;
	std::ifstream file;
	file.open(f, std::ios::in | std::ios::binary);
	file.seekg(0 - int(sizeof(BarFileFoot)), std::ios::end);

	file.read((char *)foot, sizeof(BarFileFoot));
	int rowcnt = (int(file.tellg()) - foot->footsize) / foot->rowsize;
	if (bitmap)
	{
		bitmap->resize((foot->footsize - sizeof(BarFileFoot)) * 8);
		file.seekg(-foot->footsize, std::ios::end);
		file.read((char *)bitmap->data(), foot->footsize - sizeof(BarFileFoot));
	}
	if (bars)
	{
		file.seekg(0, std::ios::beg);
		for (int i = 0; i < rowcnt; i++)
		{
			uint32_t date;
			std::vector<MinuteBar> bar;
			file.read((char *)&date, 4);
			bar.resize(foot->rowsize / sizeof(MinuteBar));
			file.read((char *)bar.data(), foot->rowsize - 4);
			std::swap((*bars)[date], bar);
		}
	}
	file.close();
}

void MyFileStore::read_bitmap_foot(fs::path f, dynamic_bitset<> *bitmap, BarFileFoot *foot)
{
	if (!foot)
		return;
	std::ifstream file;
	file.open(f, std::ios::in | std::ios::binary);
	file.seekg(0 - int(sizeof(BarFileFoot)), std::ios::end);

	file.read((char *)foot, sizeof(BarFileFoot));
	//int rowcnt = (int(file.tellg()) - foot->footsize) / foot->rowsize;
	if (bitmap)
	{
		bitmap->resize((foot->footsize - sizeof(BarFileFoot)) * 8);
		file.seekg(-foot->footsize, std::ios::end);
		file.read((char *)bitmap->data(), foot->footsize - sizeof(BarFileFoot));
	}

	file.close();
}

/*
inline bool MyFileStore::test_bit_set(uint8_t* bitmap, int bitmap_len, int index)
{
	if( index >= bitmap_len*8)
		return false;
	return bitmap[index / 8] & (1 << (index % 8));
}
*/
/*
inline int MyFileStore::count_bit(uint8_t* bitmap, int index)
{
	uint8_t* pos = bitmap;
	int b64 = index / 64;
	int bitcnt = 0;
	//Bits counting algorithm (Brian Kernighan) in an integer
	if (b64 > 0) {
		uint64_t tmp;
		for (int i=0; i<b64; i++) {
			tmp = *((uint64_t*)(pos));
			while (tmp != 0)
			{
				tmp &= (tmp - 1);
				bitcnt++;
			}
			pos += 8;
		}
		index -= 64 * b64;
	}
	if (index % 64 >= 32) {
		uint32_t tmp=* ((uint32_t*)(pos));

			while (tmp != 0)
			{
				tmp &= (tmp - 1);
				bitcnt++;
			}
			pos += 4;
			index -= 32;
	}
	
	if(index % 32 >= 16) {
		uint16_t tmp = *((uint16_t*)(pos));

		while (tmp != 0)
		{
			tmp &= (tmp - 1);
			bitcnt++;
		}
		pos += 2;
		index -= 16;
	}
	if (index % 16 >= 8) {
		uint8_t tmp = *((uint8_t*)(pos));

		while (tmp != 0)
		{
			tmp &= (tmp - 1);
			bitcnt++;
		}
		pos += 1;
		index -= 8;
	}
	int lastbit = index % 8;
	if (lastbit > 0) {
		uint8_t tmp = *((uint8_t*)(pos));
		for (int i = 0; i < lastbit; i++) {
			if (tmp & 1)bitcnt++;
			tmp >>= 1;
		}
	}
	return bitcnt;
}
*/

std::list<uint32_t> MyFileStore::read_trading_date()
{
	std::list<uint32_t> v;
	auto fpath = fs::path(dir) / trading_date_fn;
	if (!fs::exists(fpath))
		return v;
	std::ifstream f(fpath);
	std::string row;
	while (std::getline(f, row))
	{
		v.push_back(std::stoi(row));
	}

	return v;
}
std::unordered_map<uint32_t, int32_t> MyFileStore::local_trading_date(std::string symbol)
{
	auto v = std::unordered_map<uint32_t, int32_t>();
	if (symbol == "-")
		symbol = "000001";
	auto b = read_bytes((Exchange::SH), symbol, BarSize::Day1);
	int rowcnt = b.size() / row_size(BarSize::Day1);
	if (rowcnt < 1)
	{
		std::cout << "error read trading date" << std::endl;
		return v;
	}
	v.reserve(rowcnt);
	trading_range.first = 20990101;
	trading_range.second = 0;
	for (uint32_t i = 0; i < rowcnt; i++)
	{
		uint32_t day = *(uint32_t *)(b.data() + row_size(BarSize::Day1) * i);
		v[day] = i;
		if (day < trading_range.first)
			trading_range.first = day;
		if (day > trading_range.second)
			trading_range.second = day;
	}

	return v;
}
MyFileStore::~MyFileStore()
{
	//if (tradingdate.size() < 7000)
	//	return;
	inst = nullptr;
}
MyFileStore::MyFileStore(std::string dir) : FileStoreBase(dir)
{
	//update_trading_date();
	if (inst)
		std::terminate();
}
MyFileStore *MyFileStore::instance()
{
	if (!inst)
	{
		auto path = AppGlobal::instance()->get_config("bar_path");
		if (path.empty())
		{
			std::cout << "bar_path not found in config file.\n";
			std::terminate();
		}
		inst = new MyFileStore(path);
	}
	return inst;
}
void MyFileStore::start_import()
{
	if (!tdx)
	{
		std::terminate();
	}

	Exchange exchangelist[] = {Exchange::SZ, Exchange::SH};
	BarSize bss[] = {BarSize::Day1, BarSize::Min1};
	//Exchange exchangelist[] = { Exchange::SZ };
	//BarSize bss[] = { BarSize::Min1 };

	for (auto e : exchangelist)
	{

		for (auto bs : bss)
		{
			//LOG(INFO) << "import market: " << Exchange_code[int(e)]<<",BarSize:"<<BarSize_code[int(bs)];
			for (auto &p : tdx->symbol_iterator(e, bs))
			{
				auto dotpos = p.path().filename().string().find(".");
				std::string symbol = p.path().filename().string().substr(2, dotpos - 2);
				if (bs == BarSize::Day1)
				{

					std::vector<Bar> tdxbars = tdx->get_daybar(e, symbol);
					std::terminate();
					//update_daybar(e, symbol, tdxbars.data(),tdxbars.size());
				}
				else
				{

					auto tdxbars = tdx->get_minutebar(e, symbol);
					update_minutebar(e, symbol, &tdxbars);
				}
			}
		}
	}
}

//从tdx目录读取 000001日线，并扩充本地对应文件。更新 trading date
//
bool MyFileStore::update_trading_date()
{
	/*
	auto datelist = read_trading_date();
	if (datelist.size()>3000&& *datelist.begin() == 19901219 && *datelist.rbegin() == today())
		return true;

	if (datelist.size() > 3000 && *datelist.begin() <= 20050101) {
		PyRunner py;
		LOG(INFO) << "Query trading date list from yahoo";
		auto tomerge = py.get_trading_dates(*datelist.rbegin());
		LOG(INFO) << "Query end,date count:" << tomerge.size();
		for (int i = 0; i < tomerge.size(); i++) {
			if(tomerge[i]!=*datelist.rbegin())
				datelist.push_back(tomerge[i]);

		}
	}else
	{
		PyRunner py;
		LOG(INFO) << "Query trading date list from jrj website";
		auto tomerge = py.get_trading_dates();
		LOG(INFO) << "Query end,date count:" << tomerge.size();
		datelist.clear();
		for (int i = 0; i < tomerge.size(); i++) {
			
				datelist.push_back(tomerge[i]);
		}
	}

	int i = 0;
	tradingdate.clear();
	tradingdate.reserve(datelist.size());
	for (auto it = datelist.begin(); it != datelist.end(); it++) {
		tradingdate[*it] = i;
		i++;
	}
	trading_range.first = *datelist.begin();
	trading_range.second = *datelist.rbegin();
	save_trading_date();
	*/
	return true;
}
std::map<uint32_t, Bar> MyFileStore::unpack_daybar(std::vector<char> *bytes)
{
	std::map<uint32_t, Bar> v;
	int rowcnt = bytes->size() / row_size(BarSize::Day1);
	for (int i = 0; i < rowcnt; i++)
	{
		Bar bar = *(Bar *)(bytes->data() + i * row_size(BarSize::Day1));
		uint32_t date = bar.time;
		v[date] = std::move(bar);
	}
	return v;
}
bool MyFileStore::update_daybar(Exchange e, std::string symbol, Bar *b, int cnt)
{
	if (cnt == 0)
		return true;
	auto localpath = get_filepath(e, symbol, BarSize::Day1);
	auto tmppath = append_tmp(localpath);
	if (fs::exists(localpath) && fs::file_size(localpath) < sizeof(Bar))
		fs::remove(localpath);

	if (!fs::exists(localpath))
	{
		std::fstream of;
		of.open(tmppath, std::ios::out | std::ios::binary | std::ios::trunc);
		of.write((char *)(b), sizeof(Bar) * cnt);
		auto bitmap = get_bitmap(b, cnt);
		BarFileFoot foot;
		update_foot(b, cnt, &bitmap, &foot);
		write_bitmap_foot(of, &bitmap, &foot);
		of.close();
		fs::rename(tmppath, localpath);
		return true;
	}

	BarFileFoot foot;
	std::map<uint32_t, Bar> mybars;
	std::map<uint32_t, Bar>::iterator bar_it(mybars.end());
	dynamic_bitset<> bitmap;
	read_daybar(localpath, &mybars, &bitmap, &foot);

	for (auto i = b; i != b + cnt; ++i)
	{

		if (mybars.count(i->time))
			continue;
		mybars[i->time] = *i;
		if (mybars.end() == bar_it)
		{
			bar_it = mybars.find(i->time);
		}
	}
	if (mybars.end() == bar_it)
		return true;
	auto rowstart = bitmap.count(DateTime::diff_days(foot.firstdate, bar_it->first));
	bitmap = get_bitmap(&mybars);
	std::fstream of;
	fs::rename(localpath, tmppath);
	of.open(tmppath, std::ios::out | std::ios::in | std::ios::binary);
	of.seekp(rowstart * foot.rowsize);
	for (; bar_it != mybars.end(); bar_it++)
	{
		of.write((char *)&(bar_it->second.time), sizeof(Bar));
	}
	update_foot(&mybars, &bitmap, &foot);
	write_bitmap_foot(of, &bitmap, &foot);
	of.close();
	fs::rename(tmppath, localpath);
	return true;
}
bool MyFileStore::update_daybar(Exchange e, std::string symbol, std::set<Bar> b)
{
	if (b.size() == 0)
		return false;

	auto localpath = get_filepath(e, symbol, BarSize::Day1);
	auto tmppath = append_tmp(localpath);

	if (fs::exists(localpath) && fs::file_size(localpath) < sizeof(Bar))
		fs::remove(localpath);

	BarFileFoot foot;
	std::set<Bar> mybars;
	//std::set<Bar>::iterator bar_it(mybars.end());
	dynamic_bitset<> bitmap;

	if (!fs::exists(localpath))
	{
		std::fstream of;
		of.open(tmppath, std::ios::out | std::ios::binary | std::ios::trunc);
		for (auto &i : b)
		{
			of.write((char *)(&i.time), sizeof(Bar));
		}

		auto bitmap = get_bitmap(b);
		BarFileFoot foot;
		update_foot(b, &bitmap, &foot);
		write_bitmap_foot(of, &bitmap, &foot);
		of.close();
		fs::rename(tmppath, localpath);
		return true;
	}

	read_daybar_all(localpath, &mybars, &bitmap, &foot);
	int insert_time = 0;
	for (auto &i : b)
	{

		if (mybars.count(i))
			continue;
		if (insert_time == 0)
		{
			insert_time = i.time;
		}
		mybars.emplace(std::move(i));
	}
	if (insert_time == 0) // nothing to update
		return true;
	auto rowstart = bitmap.count(DateTime::diff_days(foot.firstdate, insert_time));
	bitmap = get_bitmap(mybars);
	std::fstream of;
	fs::rename(localpath, tmppath);
	of.open(tmppath, std::ios::out | std::ios::in | std::ios::binary);
	of.seekp(rowstart * foot.rowsize);
	for (auto &i : mybars)
	{
		of.write((char *)&i.time, sizeof(Bar));
	}
	update_foot(mybars, &bitmap, &foot);
	write_bitmap_foot(of, &bitmap, &foot);
	of.close();
	fs::rename(tmppath, localpath);
	return true;

	/*
	std::pair<uint32_t, uint32_t> range;
	range.first = b->at(0).date;
	range.second = b->back().date;
	int src_start=-1, src_cnt;
	if (range.second > foot.lastdate) {
		//后面补足
		
		if (range.first <= foot.lastdate) {
			//时间有交叉
			int i = b->size() - 1;
			for (; i > -1; i--) {
				if (b->at(i).date <= foot.lastdate) {
					src_start = i + 1;
					src_cnt = b->size() - i - 1;
					break;
				}
			}

		}
		else {//时间没有交叉
			
			
				//新序列刚好放置在下个交易日
				if (tradingdate[range.second] == tradingdate.find(localrange.second)->second + 1)
				{
					src_start = 0;
					src_cnt = b->size() ;
				
				}
				else {
					LOG(WARNING) << std::format("source day bar not continues with old:{},{},local:({},{}),from:({},{})", \
						Exchange_code[int(e)], symbol, localrange.first, localrange.second, range.first, range.second);
					//新序列和老序列之间不连续，可能存在停牌？
					return false;
				}
		}



		std::ofstream of;
		fs::rename(localpath, tmppath);
		of.open(tmppath, std::ios::out | std::ios::binary | std::ios::app);
		of.write((char*)(b->data() + src_start), sizeof(DayBar) * src_cnt);
		of.close();
		fs::rename(tmppath, localpath);
		return true;
	};
	if (range.first < localrange.first) {
		//前面补足
		if (range.second >= localrange.first) {
			int i = b->size() - 1;
			for (; i > -1; i--) {
				if (b->at(i).date < localrange.first) {
					src_start = 0;
					src_cnt = i + 1;
					break;
				}
			}

		}
		else {
			//新序列刚好放置在下个交易日
			if (tradingdate[range.second] + 1 == tradingdate.find(localrange.first)->second)
			{
				src_start = 0;
				src_cnt = b->size();

			}
			else {
				LOG(WARNING) << std::format("source day bar not continues with old:{},{},local:({},{}),from:({},{})", \
					Exchange_code[int(e)], symbol, localrange.first, localrange.second, range.first, range.second);
				//新序列和老序列之间不连续，可能存在停牌？
				return false;
			}
		}

		auto bytes = read_bytes(localpath);
		std::ofstream of;
		of.open(tmppath, std::ios::out | std::ios::binary | std::ios::trunc);
		of.write((char*)(b->data()), sizeof(DayBar) * src_cnt);
		of.write(bytes.data(), bytes.size());
		of.close();
		fs::remove(localpath);
		fs::rename(tmppath, localpath);
		return true;


	};
	*/
	return false;
}
void MyFileStore::read_daybar_all(fs::path f, std::set<Bar> *bars, dynamic_bitset<> *bitmap, BarFileFoot *foot)
{
	if (!foot)
		return;
	std::ifstream file;
	file.open(f, std::ios::in | std::ios::binary);
	file.seekg(0 - int(sizeof(BarFileFoot)), std::ios::end);
	file.read((char *)foot, sizeof(BarFileFoot));
	int filesize = file.tellg();
	int rowcnt = (filesize - foot->footsize) / foot->rowsize;
	if (bitmap)
	{
		bitmap->resize((foot->footsize - sizeof(BarFileFoot)) * 8);
		file.seekg(-foot->footsize, std::ios::end);
		file.read((char *)bitmap->data(), foot->footsize - sizeof(BarFileFoot));
	}
	if (bars)
	{
		file.seekg(0, std::ios::beg);
		for (int i = 0; i < rowcnt; i++)
		{
			Bar v;
			file.read((char *)&v, sizeof(Bar));
			//file.seekg(-4, std::ios::cur);
			//file.read((char*)&(*bars)[date].time, foot->rowsize);
			bars->emplace(std::move(v));
		}
	}
	file.close();
}
void MyFileStore::read_daybar(fs::path f, std::map<uint32_t, Bar> *bars, dynamic_bitset<> *bitmap, BarFileFoot *foot)
{
	if (!foot)
		return;
	std::ifstream file;
	file.open(f, std::ios::in | std::ios::binary);
	file.seekg(0 - int(sizeof(BarFileFoot)), std::ios::end);
	file.read((char *)foot, sizeof(BarFileFoot));
	int filesize = file.tellg();
	int rowcnt = (filesize - foot->footsize) / foot->rowsize;
	if (bitmap)
	{
		bitmap->resize((foot->footsize - sizeof(BarFileFoot)) * 8);
		file.seekg(-foot->footsize, std::ios::end);
		file.read((char *)bitmap->data(), foot->footsize - sizeof(BarFileFoot));
	}
	if (bars)
	{
		file.seekg(0, std::ios::beg);
		for (int i = 0; i < rowcnt; i++)
		{
			uint32_t date;
			file.read((char *)&date, 4);
			file.seekg(-4, std::ios::cur);
			file.read((char *)&(*bars)[date].time, foot->rowsize);
		}
	}
	file.close();
}
//start and end date are included.
std::string_view MyFileStore::get_daybar(Exchange exchange, std::string symbol, BarSize bs, int starttime, int endtime, int *rowsize, int *rowcnt)
{
	std::string_view bytes(nullptr, 0);
	auto f = get_filepath(exchange, symbol, bs);
	if (!check_file_valid(f))
		return bytes;
	std::ifstream file;
	BarFileFoot foot;
	file.open(f, std::ios::in | std::ios::binary);
	file.seekg(0 - int(sizeof(BarFileFoot)), std::ios::end);
	file.read((char *)&foot, sizeof(BarFileFoot));
	//int filesize = file.tellg();

	if (starttime > endtime || starttime > foot.lastdate || endtime < foot.firstdate)
	{
		file.close();
		return bytes;
	}
	if (starttime < foot.firstdate)
		starttime = foot.firstdate;
	if (endtime > foot.lastdate)
		endtime = foot.lastdate;

	if (rowsize)
		*rowsize = foot.rowsize;

	dynamic_bitset<> bitmap;
	bitmap.resize((foot.footsize - sizeof(BarFileFoot)) * 8);
	file.seekg(-foot.footsize, std::ios::end);
	file.read((char *)bitmap.data(), foot.footsize - sizeof(BarFileFoot));
	auto index1 = bitmap.count(DateTime::diff_days(foot.firstdate, starttime));
	int dayindex2 = DateTime::diff_days(foot.firstdate, endtime);
	auto index2 = bitmap.count(dayindex2);
	file.seekg(index1 * foot.rowsize, std::ios::beg);

	int cnt;

	if (bitmap.test(dayindex2))
		cnt = index2 - index1 + 1;
	else
		cnt = index2 - index1;

	int size = cnt * foot.rowsize;
	char *buf = (char *)malloc(size);

	file.read(buf, size);

	if (rowcnt)
		*rowcnt = cnt;
	file.close();
	return std::string_view(buf, size);
}

std::string_view MyFileStore::get_rbar_series(Exchange exchange, std::string symbol, BarSize bs, int rindex, int rcount, int *rowsize, int *rowcnt)
{
	std::string_view bytes(nullptr, 0);
	auto f = get_filepath(exchange, symbol, bs);
	if (!check_file_valid(f))
		return bytes;
	std::ifstream file;
	BarFileFoot foot;
	file.open(f, std::ios::in | std::ios::binary);
	file.seekg(0 - int(sizeof(BarFileFoot)), std::ios::end);
	file.read((char *)&foot, sizeof(BarFileFoot));
	int filesize = file.tellg();
	int total_rows = (filesize - foot.footsize) / foot.rowsize;

	if (rindex >= total_rows)
	{
		file.close();
		return bytes;
	}
	if (rindex + rcount > total_rows)
		rcount = total_rows - rindex;
	if (rowcnt)
		*rowcnt = rcount;
	if (rowsize)
		*rowsize = foot.rowsize;

	file.seekg((total_rows - rcount - rindex) * foot.rowsize, std::ios::beg);
	int size = rcount * foot.rowsize;
	void *buf = malloc(size);
	//bytes.resize(rcount * foot.rowsize);

	file.read((char *)buf, size);

	file.close();
	return std::string_view((char *)buf, size);
}

std::vector<Bar> MyFileStore::get_daybar(Exchange exchange, std::string symbol)
{
	std::vector<Bar> v;
	std::map<uint32_t, Bar> bars;
	auto p = get_filepath(exchange, symbol, BarSize::Day1);
	BarFileFoot foot;
	read_daybar(p, &bars, nullptr, &foot);
	v.resize(bars.size());
	int idx = 0;
	for (auto &i : bars)
	{
		v[idx++] = i.second;
	}
	return v;
}
bool MyFileStore::update_minutebar(Exchange e, std::string symbol, std::map<uint32_t, std::vector<MinuteBar>> *bars)
{
	if (bars->size() == 0)
		return false;

	auto localpath = get_filepath(e, symbol, BarSize::Min1);
	auto localpath_tmp = append_tmp(localpath);

	BarFileFoot foot;
	if (fs::exists(localpath) && fs::file_size(localpath) < sizeof(uint32_t) + 240 * sizeof(MinuteBar))
		fs::remove(localpath);

	if (!fs::exists(localpath))
	{
		auto bitmap = get_bitmap(bars);
		update_foot(bars, &bitmap, &foot);
		std::fstream of;
		of.open(localpath_tmp, std::ios::binary | std::ios::out | std::ios::trunc);
		write_bars(of, bars, bars->begin());
		write_bitmap_foot(of, &bitmap, &foot);
		of.close();
		fs::rename(localpath_tmp, localpath);
		return true;
	}
	else
	{ //需要与现有文件合并

		std::map<uint32_t, std::vector<MinuteBar>> localbars;
		std::map<uint32_t, std::vector<MinuteBar>>::iterator bar_it(localbars.end());
		dynamic_bitset<> bitmap;
		read_minutebar(localpath, &localbars, &bitmap, &foot);

		for (auto &i : *bars)
		{
			if (localbars.count(i.first))
				continue;

			localbars[i.first] = std::move(i.second);
			if (bar_it == localbars.end())
			{
				bar_it = localbars.find(i.first);
			}
		}

		if (bar_it == localbars.end())
			return true;

		auto index = DateTime::diff_days(foot.firstdate, bar_it->first);
		auto write_rows = bitmap.count(index);

		bitmap = get_bitmap(&localbars);

		std::fstream of;
		fs::rename(localpath, localpath_tmp);
		of.open(localpath_tmp, std::ios::out | std::ios::in | std::ios::binary);
		of.seekp(write_rows * foot.rowsize, std::ios::beg);
		write_bars(of, &localbars, bar_it);
		update_foot(&localbars, &bitmap, &foot);
		write_bitmap_foot(of, &bitmap, &foot);
		of.close();
		fs::rename(localpath_tmp, localpath);
		return true;
	}
}
std::map<uint32_t, std::vector<MinuteBar>> MyFileStore::unpack_minbar(std::vector<char> *bytes, dynamic_bitset<> *bitmap, BarFileFoot *foot)
{
	if (!foot)
	{
		std::terminate();
	}
	std::map<uint32_t, std::vector<MinuteBar>> bars;

	memcpy(foot, bytes->data() + bytes->size() - sizeof(BarFileFoot), sizeof(BarFileFoot));
	if (bitmap)
	{
		bitmap->resize((foot->footsize - sizeof(BarFileFoot)) * 8);
		memcpy(bitmap->data(), bytes->data() + bytes->size() - foot->footsize, foot->footsize - sizeof(BarFileFoot));
	}

	int rowcnt = (bytes->size() - foot->footsize) / foot->rowsize;

	for (int i = 0; i < rowcnt; i++)
	{
		uint32_t date = *((uint32_t *)(bytes->data() + i * foot->rowsize));
		bars[date].resize((foot->rowsize - 4) / sizeof(MinuteBar));
		memcpy(bars[date].data(), bytes->data() + i * foot->rowsize + sizeof(date), foot->rowsize - sizeof(date));
	}
	return bars;
}
fs::path MyFileStore::get_filepath(Exchange exchangeid, std::string symbol, BarSize barsize)
{
	auto v = fs::path(dir);
	v /= std::string(Exchange_code[int(exchangeid)]);
	v /= std::string(BarSize_code[int(barsize)]);
	v /= symbol;
	return v;
}
uint16_t MyFileStore::row_size(BarSize barsize)
{
	if (barsize == BarSize::Day1)
		return sizeof(Bar);
	else if (barsize == BarSize::Min1)
	{
		std::cout << "You can't query min bar row size." << std::endl;
		std::terminate();
	}

	return -1;
}
